import pandas as pd
import requests
import os
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException, TimeoutException
import time

# 读取 Excel 文件中的企业名称列表
input_file = 'F:\\qcc3.xlsx'
output_file = 'F:\\企查查数据.xlsx'
# 读取 Excel 文件
df = pd.read_excel(input_file)
urls = df['url'].tolist()

# 初始化结果列表
results = []

# 设置 Chrome 选项
chrome_options = Options()

# 添加基本的请求头信息
chrome_options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
# 初始化 WebDriver
service = Service(executable_path=r'F:\chromedriver-win64\chromedriver.exe')
driver = webdriver.Chrome(service=service, options=chrome_options)

results = []

#
# # 设置复杂的请求头信息
headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
    "Accept-Encoding": "gzip, deflate, br, zstd",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
    "Cache-Control": "max-age=0",
    "Connection": "keep-alive",
    "Host": "www.cmde.org.cn",
    "Sec-Fetch-Dest": "document",
    "Sec-Fetch-Mode": "navigate",
    "Sec-Fetch-Site": "none",
    "Sec-Fetch-User": "?1",
    "Upgrade-Insecure-Requests": "1",
    "sec-ch-ua": "\"Google Chrome\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"",
    "sec-ch-ua-mobile": "?0",
    "sec-ch-ua-platform": "\"Windows\""
}

# 使用 CDP 命令设置请求头
driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
    "source": """
            Object.defineProperty(navigator, 'webdriver', {
              get: () => undefined
            })
          """
})

# 创建下载目录
download_dir = r'F:\knowledge'
if not os.path.exists(download_dir):
    os.makedirs(download_dir)

# 爬取数据
for url in urls:
    driver.get(url)
    try:
        # 等待元素加载
        wait = WebDriverWait(driver, 15)  # 最多等待 15 秒
        result_div = wait.until(EC.presence_of_element_located((By.XPATH, "//div[@class='wenzhang w1200-auto']")))
        # 查找第一个搜索结果
        if result_div:
            title_tag = result_div.find_element(By.CSS_SELECTOR, 'h2.title').text.strip()
            two_title_tag = result_div.find_element(By.CSS_SELECTOR, 'h3.two-title').text.strip()
            content_tag = result_div.find_element(By.CSS_SELECTOR, 'div.text').text.strip()
            date_tag = result_div.find_element(By.CSS_SELECTOR, 'div.date').text.strip()
            # 检查附件是否存在
            try:
                attach_tag = result_div.find_element(By.CSS_SELECTOR, 'div.text > p:nth-child(7) > a')
                attach_name = attach_tag.text
                attach_link = attach_tag.get_attribute('href')

                # 下载附件
                response = requests.get(attach_link, headers=headers)
                if response.status_code == 200:
                    file_path = os.path.join(download_dir, attach_name)
                    with open(file_path, 'wb') as f:
                        f.write(response.content)
                    results.append({
                        '地址': url,
                        '标题': title_tag,
                        '副标题': two_title_tag,
                        '内容': content_tag,
                        '发布日期': date_tag,
                        '附件名称': attach_name,
                        '附件链接': attach_link,
                        '本地路径': file_path
                    })
                else:
                    results.append({
                        '地址': url,
                        '标题': title_tag,
                        '副标题': two_title_tag,
                        '内容': content_tag,
                        '发布日期': date_tag,
                        '附件名称': attach_name,
                        '附件链接': attach_link,
                        '本地路径': None
                    })
            except NoSuchElementException:
                results.append({
                    '地址': url,
                    '标题': title_tag,
                    '副标题': two_title_tag,
                    '内容': content_tag,
                    '发布日期': date_tag,
                    '附件名称': None,
                    '附件链接': None,
                    '本地路径': None
                })
        else:
            results.append({'地址': url, '备注': '未找到相关信息'})
    except TimeoutException:
        results.append({'地址': url, '备注': '请求超时'})
    except Exception as e:
        results.append({'地址': url, '备注': f'请求失败，错误信息: {str(e)}'})

    # 设置请求间隔，避免被封禁
    time.sleep(1)

# 关闭 WebDriver
driver.quit()

# 将结果写入新的 Excel 文件
result_df = pd.DataFrame(results)
result_df.to_excel(output_file, index=False)

print(f'数据已保存到 {output_file}')
